package Java网络爬虫;

import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;

/**
 * @Author tyy
 * @Description TODO
 * @Since 2021/11/27
 */
public class ImageCrawl {
    private static String url = "https://www.nipic.com/topic/show_27202_1.html";
    public static void main(String[] args) throws IOException {
//        apacheHttpClient();
        Document document = Jsoup.connect(url).get();
//        Document document = Jsoup.parse(url);
        Elements elements = document.select("li.new-search-works-item");
        for (int i = 0; i < elements.size(); i++) {
            Elements imgElements = elements.get(i).select("a>img");
            Connection.Response response = Jsoup.connect("https:"+imgElements.attr("src"))
                    .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36\"")
                    .ignoreContentType(true).execute();
            String name = imgElements.attr("alt");
            ByteArrayInputStream stream = new ByteArrayInputStream(response.bodyAsBytes());
            FileUtils.copyInputStreamToFile(stream, new File("E:/data/"+ name+i+".png"));


        }



    }

    private static void apacheHttpClient() {
        HttpClient client = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(url);
        httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36");
        HttpResponse response = null;
        try {
            response = client.execute(httpGet);
            HttpEntity entity = response.getEntity();
            System.out.println("entity = " + EntityUtils.toString(entity));

        } catch (IOException e) {
            e.printStackTrace();
        }

    }
}
